% scribe: Richard Crump
% lastupdate: 6 December 2005
% lecture: 21
% references: Durrett, Section 4.1
% title: Conditional Distributions
% keywords: conditional distribution, regular conditional distribution, Markov kernel, Bayes' rule, Generalized Fubini's theorem, reference measures
% end

\documentclass[12pt, letterpaper]{article}

\include{macros}

\begin{document}

\lecture{21}{Conditional Distributions}{Richard Crump}{crump@econ.berkeley.edu}

\section{Related Ideas}
% keywords: conditional distribution, regular conditional distribution, Markov kernel, Bayes' rule, Generalized Fubini's theorem, reference measures
% end

\begin{enumerate}
\item Regular conditional distribution for $Y$ given $X$ (or a sigma field $\sigma(X)$).
\item Markov kernel (transition function/operator).
\end{enumerate}

\subsection{Setup, Definitions and Terminology}
% Keywords: Conditional distribution, Regular conditional distribution, Markov kernel, Bayes' rule, Generalized Fubini's theorem.
% end

Two measurable spaces: $(S, \mathcal{S})$ and $(T, \mathcal{T})$ (very often we have $S=T$ and $\mathcal{S}=\mathcal{T}$).
\begin{definition}
A \emph{Markov kernel} $P(s,A)$ from $(S, \mathcal{S})$ to $(T, \mathcal{T})$ is a collection of probability measures on $(T, \mathcal{T})$ indexed in an $\mathcal{S}$-measurable way by a parameter $s \in S$, satisfying:
\begin{enumerate}
\item[i.] for each $s \in S$, $P(s, \cdot)$ is a probability measure
  on $(T, \mathcal{T})$; and
\item[ii.] for each $A \in \mathcal{T}$, $s \mapsto P(s,A)$ is measurable relative to $\mathcal{S}$.
\end{enumerate}
\end{definition}

\begin{example} 
From the theoretical statistics literature:  $\P_\theta \,\,\, \theta
\in \Theta$.  Where $\P_\theta$ are defined on the measurable space
$(X, \mathcal{X}$).
\end{example}

We now introduce the idea of a regular conditional distribution.
\begin{definition}
For a pair of random variables $(X,Y)$, $X:\Omega \mapsto S$ and
$Y:\Omega \mapsto T$, defined on a background space $(\Omega, \F,
\P)$, we say that a Markov kernel serves as a \emph{regular conditional distribution} for $Y$ given $X$ if and only if the joint distribution of $(X,Y)$ is given by the formula
\[ \P(X \in dx, Y \in dy) = \P(X \in dx) \cdot P(x, dy). \]
\end{definition}

This means (by definition!) that
\begin{eqnarray}
\P(X \in A, Y \in B) 
&=& \int_A \P(X \in dx) \cdot P (x, B) \label{eq:rcd1} \\
&=& \int_S \1_{(x \in A)} \cdot \P(X \in dx) \cdot P(x, B) \nonumber \\
&=& \E \left[ \1_{(X \in A)} \cdot P(X,B) \right]. \nonumber
\end{eqnarray}

Take $g(\cdot) \geq 0$ and product measurable: then,
\begin{eqnarray} 
\E \left[ g(X, Y) \right] 
&=& \int_S \P(X \in dx) \left[ \int_T P(x, dy) g(x, y) \right] \label{eq:rcd2} \\
&=& \int_S \left[ \int_T g(x,y) \cdot P(x, dy) \right] \P(X \in dx). \nonumber
\end{eqnarray}
\begin{theorem}[Generalized Fubini's Theorem] (\ref{eq:rcd1}) holds for all $A \in \mathcal{S}, \,\, B \in \mathcal{T}$ if and only if (\ref{eq:rcd2}) holds for all $g \geq 0$ product measurable (and hence for signed $g$, provided result is finite with $g \mapsto |g|$).
\end{theorem}

\subsection{Examples}
% keywords: reference measures.
% end

\begin{example}
If $X$, $Y$ have density $f(x,y)$ relative to $dx$, $dy$  for some reference measures $dx$ on $(S, \mathcal{S})$ and $dy$ on $(T, \mathcal{T})$, then
\[ P(x, dy) = \frac{f(x,y)}{f_X(x)} dy \]
In this case it is easy to flip and get a conditional distribution of $X$ given $Y$, that is 
\[ \hat P(y, dx) = \frac{f(x,y)}{f_Y(y)} dx \]
by Bayes' Rule.  But in a general setting (setting of Generalized Fubini's Theorem cannot do this),  there is no Bayes' formula without reference measures.  But, there is a general theory which says that provided $(S, \mathcal{S})$ is a nice, measurable space (\cite{durrett}, chapter 4, (1.6))
then there does exist a Markov kernel $\hat P (y, dx)$ so that 
\[ \P(X \in dx, Y \in dy) = \P(Y \in dy) \cdot \hat P(y, dx) \]
in the same sense as before.
\end{example}

\begin{example} 
A point $(X,Y)$ is picked proportional to length measure on the perimeter of an equilateral triangle with vertices $(0,0)$, $(1,0)$ and $(\frac{1}{2}, \frac{\sqrt{3}}{2})$.  Note that $(X,Y)$ is easily defined on $(\Omega, \F, \P) = ([0,1], \borel, \lambda)$.  

Observe that $X \sim \mathcal{U}[0,1]$ and $Y \sim \frac{1}{3} \cdot \delta_0 + \frac{2}{3} \cdot \mathcal{U}[0, \frac{\sqrt 3}{2}]$ by inspection.  What is the conditional distribution of $Y$ given $X=x$?  

Note first that for $0 \leq x \leq \frac{1}{2}$ the only possible values of Y consistent with $X=x$ are $y=0$ or $y=\sqrt 3 x$.  Thus
\[ \P(Y>0 | 0 \leq x \leq \frac{1}{2}) = \frac{\P(Y>0, 0 \leq x \leq \frac{1}{2})}{\P(0 \leq x \leq \frac{1}{2})} = \frac{\frac{1}{3}}{\frac{1}{3} + \frac{1}{6}} = \frac{2}{3} \]
More generally, choose $\varepsilon > 0$ and $0 < x < 1-\varepsilon$.  Then,
\[ \P(Y>0 | x < X < x+ \varepsilon) = \frac{\P(Y>0, x < X < x + \varepsilon)}{\P(x < X < x+ \varepsilon)} = \frac{\frac{2}{3}\varepsilon}{\varepsilon} = \frac{2}{3} \]
and 
\[ \P(Y=0|x < X < x+ \varepsilon) = \frac{1}{3} \]
So for $X=x$ where $0 \leq x \leq \frac{1}{2}$ the conditional distribution is 
\begin{displaymath}
(Y|X=x) = \left\{ \begin{array}{ll} 0 & \textrm{with probability} \,\, \frac{1}{3} \\
& \\
\sqrt 3 x & \textrm{with probability} \,\, \frac{2}{3} \end{array} \right.
\end{displaymath}
A similar argument follows for $\frac{1}{2} \leq x \leq 1$ with a
different line.  
\end{example}

\begin{proof}
You may check the details.  Just check $\P(X \in A, Y \in B)$ is given by integration of this regular conditional distribution.  Compute the lengths of intervals.
\end{proof}

\textbf{Question:} How does this relate to conditional expectation?  

\textbf{Answer:} Suppose $P(x, \cdot)$ gives a regular conditional distribution for $Y$ given $X=x$ as defined before, i.e., $\P(X \in x, Y \in dy) = \P(X \in x) \cdot P(x, dy)$ then we can compute
\[ \E[ h(Y) | X=x] = \int_S P(x, dy) \cdot h(y) \equiv \psi(x) \]
so that
\[ \E[ h(Y) | X] = \psi(X) \]
and
\[ \E[ h(Y) ] = \E[ \psi(X) ] \]

\bibliographystyle{plain}
\bibliography{../books.bib}
\end{document}

